#include "PageLib.hpp"
#include "tinyxml2.h"
#include "WebPage.hpp"
#include <iostream>
#include <string>
#include <regex>
#include <vector>
#include <fstream>
#include <map>

using std::cout;
using std::endl;
using std::string;
using std::vector;
using std::ofstream;
using std::map;

using namespace tinyxml2;


PageLib::PageLib(Configuration* conf, 
                 DirScanner& dirScanner)
                 /* FileProcessor& fileProcessor) */
: _dirScanner(dirScanner)
, _conf(conf)
{
    map<string, string> mp = conf->getConfigMap();
    string xmlFilesPath = mp.find("xmlFiles")->second;
    /* cout << "xmlFilesPath: " << xmlFilesPath << endl; */

    _dirScanner(xmlFilesPath);
    _files = _dirScanner.files();

}



void PageLib::create(){
    int idx = 1;
    CppJieba jieba;
    cout << "jieba" <<endl;

    for(auto& file: _files){
        XMLDocument doc;
        doc.LoadFile(file.c_str());
        if(doc.ErrorID())
        {
            std::cerr << "loadFile fail" << endl;
            return;
        }


        XMLElement *itemNode = 
            doc.FirstChildElement("rss")->FirstChildElement("channel")->FirstChildElement("item");
        while(itemNode)
        {
            string title = itemNode->FirstChildElement("title")->GetText();
            /* cout << "title: " << title << endl; */
            string url=  itemNode->FirstChildElement("link")->GetText();
            /* cout << "url: " << url << endl; */
            // 如果没有content就用description代替
            string content;
            if(itemNode->FirstChildElement("content:encoded")){
                content = itemNode->FirstChildElement("content:encoded")->GetText();
            }
            else if(itemNode->FirstChildElement("content")){
                content = itemNode->FirstChildElement("content")->GetText();
            }
            else if(itemNode->FirstChildElement("description")){
                content = itemNode->FirstChildElement("description")->GetText();
            }
            else{// 没有内容，直接跳过这篇文章
                itemNode = itemNode->NextSiblingElement("item");
                continue;
            }


            std::regex reg("<[^>]+>");//通用正则表达式过滤html
            content = regex_replace(content, reg, "");
            /* cout << "content: " << content << endl; */
        
            WebPage wp(_conf, jieba, idx, title, url, content);

            _webPages.push_back(std::move(wp));

            ++idx;
            itemNode = itemNode->NextSiblingElement("item");
        }
    }
}

#if 1
void PageLib::store(){
    const string ripepagePath = "ripepage.dat";
    const string offsetPath = "offset.dat";
    ofstream ofs(ripepagePath);
    if(!ofs)
    {
        std::cerr << "open " << ripepagePath << " fail!" << endl;
        return;
    }

    ofstream ofsOffset(offsetPath);
    if(!ofsOffset)
    {
        std::cerr << "open " << offsetPath << " fail!" << endl;
        return;
    }

    size_t begin, end;
    cout << "PageLib store" << _webPages.size()<< endl;
    for(size_t idx = 0; idx != _webPages.size(); ++idx)
    {
        ofs << "<doc>" << endl 
            << "<docid>" << idx + 1 << "</docid>" << endl 
            << "<title>" << _webPages[idx].getTitle() << "</title>" << endl 
            << "<url>" << _webPages[idx].getUrl() << "</url>" << endl 
            << "<content>"; 
        // 写入ripepage.dat的同时设置偏移库offset.dat
        
        begin = ofs.tellp();
        ofsOffset << idx + 1 << " " << begin << " ";

        ofs << _webPages[idx].getContent(); 

        end = ofs.tellp();
        ofsOffset << end - begin << endl;
        
        ofs << "</content>" << endl 
            << "</doc>" << endl;
        /* cout << ofs.tellp() << endl; */
    }

    ofs.close();
    ofsOffset.close();

}
    
vector<WebPage>  & PageLib::getWebPages(){
    return _webPages;
}
#endif
